package wvtools;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;

import edu.udo.cs.wvtool.config.WVTConfiguration;
import edu.udo.cs.wvtool.generic.stemmer.WVTStemmer;
import edu.udo.cs.wvtool.generic.tokenizer.WVTTokenizer;
import edu.udo.cs.wvtool.util.TokenEnumeration;

import preprocess.XMLDoc;
import utils.Configuration;
import wvtools.extend.WVTXMLDocumentInfo;

/**
 * not used by the current project
 * 
 * @author huangcd
 * 
 */
public class WordListExtend
{
    public final static int         shiftSize = 32;
    public final static long        tag       = (1L << shiftSize) - 1;
    private HashMap<String, Struct> wordMap;
    private WVToolsWrapper          wvtools;
    private int                     numDocument;

    /**
     * return (index << shiftSize) | documentCount
     * 
     * @param index
     * @param documentCount
     * @return
     */
    public static long getValue(long index, long documentCount)
    {
        return (index << shiftSize) | documentCount;
    }

    /**
     * return value >> shiftSize
     * 
     * @param value
     * @return
     */
    public static long getIndex(long value)
    {
        return value >> shiftSize;
    }

    /**
     * return value & tag
     * 
     * @param value
     * @return
     */
    public static long getDocumentCount(long value)
    {
        return value & tag;
    }

    public WordListExtend() throws IOException
    {
        wvtools = WVToolsWrapper.newInstance();
        loadMap();
    }

    private void loadMap() throws IOException
    {
        wordMap = new HashMap<String, Struct>();
        File wordListPath = new File(Configuration.WordListFile);
        BufferedReader reader = new BufferedReader(new FileReader(wordListPath));
        String str;
        Struct s;
        String word;
        str = reader.readLine();
        str = str.replace("@number_of_documents ", "");
        numDocument = new Integer(str);
        reader.readLine();
        int index = 0;
        while ((str = reader.readLine()) != null)
        {
            String[] arr = str.split(",");
            try
            {
                s = new Struct();
                word = arr[0];
                double count = new Integer(arr[1]);
                s.idf = Math.log(((double) numDocument) / count);
                s.index = index;
                wordMap.put(word, s);
                index++;
            }
            catch (Exception e)
            {}
        }
    }

    public HashMap<Integer, Double> createVSMMap(XMLDoc doc)
    {
        try
        {
            HashMap<String, Integer> appearMap = new HashMap<String, Integer>();
            String text = doc.getAllText();
            WVTConfiguration config = wvtools.getDefaultConfiguration();
            WVTXMLDocumentInfo d = wvtools.createWVTXMLDocumentInfo(doc);
            // WVTCharConverter charConverter = null;
            WVTTokenizer tokenizer = null;
            // WVTWordFilter wordFilter = null;
            WVTStemmer stemmer = null;
            // charConverter = (WVTCharConverter) config.getComponentForStep(
            // WVTConfiguration.STEP_CHAR_MAPPER, d);
            tokenizer = (WVTTokenizer) config.getComponentForStep(
                    WVTConfiguration.STEP_TOKENIZER, d);
            // wordFilter = (WVTWordFilter) config.getComponentForStep(
            // WVTConfiguration.STEP_WORDFILTER, d);
            stemmer = (WVTStemmer) config.getComponentForStep(
                    WVTConfiguration.STEP_STEMMER, d);
            TokenEnumeration tokens = stemmer.stem(tokenizer.tokenize(
                    new StringReader(text), d), d);

            int numTerm = 0;
            while (tokens.hasMoreTokens())
            {
                String word = tokens.nextToken();
                if (wordMap.containsKey(word))
                {
                    int count;
                    if (appearMap.containsKey(word))
                        count = appearMap.get(word) + 1;
                    else
                        count = 1;
                    appearMap.put(word, count);
                    numTerm++;
                }
            }

            HashMap<Integer, Double> result = new HashMap<Integer, Double>();
            double length = 0;
            for (String word : appearMap.keySet())
            {
                Struct s = wordMap.get(word);
                double weight = s.idf * appearMap.get(word) / numTerm;
                result.put(s.index, weight);
                length += weight * weight;
            }
            length = Math.sqrt(length);

            if (length > 0.0)
                for (String word : appearMap.keySet())
                {
                    int index = wordMap.get(word).index;
                    result.put(index, result.get(index) / length);
                }
            return result;
        }
        catch (Exception e)
        {
            e.printStackTrace();
        }
        return null;
    }
}

class Struct
{
    double idf;
    int    index;
}
